# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy
from scrapy_redis.spiders import RedisSpider
from Books.items import JdItem
import json
import urllib
import re

class JdSpider(RedisSpider):
    name = 'jd'
    allowed_domains = ['jd.com','p.3.cn']
    # start_urls = ['https://book.jd.com/booksort.html']
    redis_key = 'jd:start_urls'

    def parse(self, response):
        # 大分类列表
        dt_list = response.xpath('//div[@class="mc"]/dl/dt')
        for dt in dt_list:
            item = JdItem()
            item['b_cate'] = dt.xpath('./a/text()').get()
            # 小分类列表
            em_list = dt.xpath('./following-sibling::dd[1]/em')
            for em in em_list:
                item['s_href'] = em.xpath('./a/@href').get()
                item['s_cate'] = em.xpath('./a/text()').get()
                if item['s_href'] is not None:
                    item['s_href'] = 'https:' + item['s_href']
                    yield scrapy.Request(
                        url=item['s_href'],
                        callback=self.parse_book_list,
                        meta={'item':deepcopy(item)}
                    )
    # 解析列表页
    def parse_book_list(self, response):
        item = response.meta['item']
        li_list = response.xpath('//div[@id="plist"]/ul/li')
        for li in li_list:
            item['book_img'] = li.xpath('.//div[@class="p-img"]//img/@src').get()
            if item['book_img'] is None:
                item['book_img'] = li.xpath('.//div[@class="p-img"]//img/@data-lazy-img').get()
            item['book_img'] = item['book_img']
            item['book_name'] = li.xpath('.//div[@class="p-name"]/a/em/text()').get().strip()
            item['book_author'] = li.xpath('.//span[@class="author_type_1"]/a/text()').getall()
            item['book_press'] = li.xpath('.//span[@class="p-bi-store"]/a/@title').get()
            item['book_publish_date'] = li.xpath('//span[@class="p-bi-date"]/text()').get().strip()
            item['book_sku'] = li.xpath('./div/@data-sku').get()
            yield scrapy.Request(
                url="https://p.3.cn/prices/mgets?skuIds=J{sku}".format(sku=item['book_sku']),
                callback=self.parse_book_price,
                meta={'item': deepcopy(item)}
            )
            cat = re.findall(r"//list\.jd\.com/(.*?)\.html",item['s_href'])[0]
            cat = re.sub("-",",",cat)
            detail_url = 'https://cd.jd.com/promotion/v2?skuId={sku}&area=17_1479_3163_70151&cat={cat}'.format(sku=item['book_sku'],cat=cat)
            if detail_url is not None:
                yield scrapy.Request(
                    url=detail_url,
                    callback=self.parse_book_detail,
                    meta={'item': deepcopy(item)}
                )

        # 列表页翻页
        next_url = response.xpath('//a[@class="pn-next"]/@href').get()
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url,next_url)
            yield scrapy.Request(
               url=next_url,
               callback=self.parse_book_list,
               meta={'item':deepcopy(item)}
            )

    def parse_book_price(self, response):
        item = response.meta['item']
        item['book_price'] = json.loads(response.body.decode('utf-8'))[0]['op']

    # 详情页信息
    def parse_book_detail(self, response):
        item = response.meta['item']
        item['book_desc'] = json.loads(response.body.decode('utf-8'))["ads"][0].get('ad')
        yield item
